#ifndef TOKENIZER_H
#define TOKENIZER_H
#include<QString>
#include<QChar>
#include<QTextStream>
class Tokenizer
{
	public:
		Tokenizer(void);
		Tokenizer(const QString& filename,QTextStream& fp)
		~Tokenizer();
		bool IsEOF();
		{
			return m_TokenIndex > m_BufferLen;
		}

		QChar CurrentChar()
		{
			return m_Buffer.at(m_TokenIndex);
		}
	
		QChar NextChar()
		{
			if(!IsEOF())
			{
				return m_Buffer.at(m_TokenIndex+1);
			}
			return QChar('\0');
		}
		
		int MoveNextChar()
		{
			m_TokenIndex++;
		}
		
		int GetLineNum()
		{
			return m_BufferLen;
		}

		QString GetFileName()
		{	
			return m_FileName;
		}

		bool SkipComment();
		bool SkipWhiteSpace();
		bool MoveNextLine();
		bool SkipBlock(const QChar& ch);
		bool SkipUnWant();
		
	private:
		QString m_Buffer;
		int m_BufferLen;
		QString m_FileName;
		int m_TokenIndex;
		int m_LineNum;
		int m_ColumnNum;   //列号
		void BaseInit();
		

};
#endif
